﻿using AngleSharp.Dom;
using AngleSharp.Html.Parser;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Polly;
using System;
using System.Collections.Generic;
using System.Text;
using System.Text.RegularExpressions;
using NLog;
using NLog.Extensions.Logging;
using System.Threading.Tasks;
using System.Threading.Tasks.Schedulers;
using System.Threading;
using System.Linq;

namespace Spider
{
    class Program
    {

        #region 下载相关
        //const string baseurl = "https://www.dy2018.com/";
        const string baseurl = "https://www.dytt89.com/";
        static RedisExchange redis = new RedisExchange();
        static int maxTask = 5;
        static List<int> numbers = new List<int>() { 1, 2, 3, 4, 5 };
        static object locker = new object();//定义对象
        static List<int> pageStart = new List<int>();
        static Random rd = new Random();
        static void Main(string[] args)
        {
            try
            {
                if (args.Length > 0)
                {
                    if (!int.TryParse(args[0], out maxTask))
                    {
                        NLogHelper.Error("线程参数应为数字，是否继续以默认线程数进行下载，Y/N?");
                        string cmd = Console.ReadLine();
                        if (cmd.ToUpper() == "N")
                        {
                            Console.WriteLine("Press any key to exit");
                            return;
                        }
                    }
                }
                var config = new ConfigurationBuilder()
                   .SetBasePath(System.IO.Directory.GetCurrentDirectory()) //From NuGet Package Microsoft.Extensions.Configuration.Json
                   .AddJsonFile("appsettings.json", optional: true, reloadOnChange: true)
                   .Build();

                var servicesProvider = BuildDi(config);
                using (servicesProvider as IDisposable)
                {
                    Console.OutputEncoding = System.Text.Encoding.Default;
                    Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
                    FindMore();
                    Console.ReadKey();
                }
            }
            catch (Exception ex)
            {
                NLogHelper.Error("Stopped program because of exception," + ex.Message);
                throw;
            }
            finally
            {
                LogManager.Shutdown();
            }
        }
        private static IServiceProvider BuildDi(IConfiguration config)
        {
            return new ServiceCollection()
               .AddTransient<Runner>() // Runner is the custom class
               .AddLogging(loggingBuilder =>
               {
                   // configure Logging with NLog
                   loggingBuilder.ClearProviders();
                   loggingBuilder.SetMinimumLevel(Microsoft.Extensions.Logging.LogLevel.Trace);
                   loggingBuilder.AddNLog(config);
               })
               .BuildServiceProvider();
        }

        public async static Task<string> GetHtmlByUrlAsync(string url)
        {
            //先执行这个，不然使用GB2312报错
            Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
            try
            {
                System.Net.WebRequest wRequest = System.Net.WebRequest.Create(url);
                wRequest.ContentType = "text/html; charset=gb2312";

                wRequest.Method = "get";
                wRequest.UseDefaultCredentials = true;
                // 获取Url返回html信息
                var task = await wRequest.GetResponseAsync();
                //System.Net.WebResponse wResp = task.Result;
                System.IO.Stream respStream = task.GetResponseStream();
                //使用GB2312编码读取
                using (System.IO.StreamReader reader = new System.IO.StreamReader(respStream, Encoding.GetEncoding("GB2312")))
                {
                    string ret = reader.ReadToEnd();
                    //Console.Write(ret);
                    return ret;
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.ToString());
                return string.Empty;
            }
        }


        //打开主页，查找更多选项
        public async static void FindMore()
        {
            try
            {
                NLogHelper.Info("寻找网站入口...");                
                var htmlSourceCode = await GetHtmlByUrlAsync(baseurl);
                if (string.IsNullOrEmpty(htmlSourceCode))
                {
                    Console.WriteLine("站点访问失败，1分钟后将重试");
                    Thread.Sleep(1000 * 60);
                    FindMore();
                }
                else
                {
                    var parser = new HtmlParser();
                    var document = parser.ParseDocument(htmlSourceCode);
                    var MoreList = document.QuerySelectorAll("em > a");
                    List<string> list = GetItemsHref(MoreList);
                    list.RemoveAt(2);//这个页面暂不解析
                    string firsturl = list[0];
                    if (!firsturl.Contains(baseurl))
                    {
                        list.RemoveAt(0);
                        list.Add(firsturl.Replace("dy2018", "dytt89"));//临时处理
                    }
                    GetPageInfo(list, parser);
                }
            }
            catch (Exception ex)
            {
                NLogHelper.Error("寻找入口出错:" + ex.Message);
            }
        }

        //解析url地址
        static List<string> GetItemsHref(IEnumerable<IElement> elements)
        {
            List<string> ret = new List<string>();
            try
            {
                NLogHelper.Info("解析网址...");
                foreach (var item in elements)
                {
                    Console.WriteLine(item.Attributes["href"]?.Value);
                    string link = item.Attributes["href"]?.Value;
                    if (!link.Contains("http"))
                        link = baseurl + link;
                    ret.Add(link);
                    NLogHelper.Info("网址:" + link + "...ok");
                }
            }
            catch (Exception ex)
            {
                NLogHelper.Error("解析出错:" + ex.Message);
            }
            return ret;
        }


        //分别进入“更多”，获取总页码，页数，条数
        static void GetPageInfo(List<string> urls, HtmlParser parser)
        {
            ISyncPolicy policy2 = Policy.Handle<Exception>()
                .Retry(3);//重试3次
            policy2.Execute(() =>
            {
                foreach (string url in urls)
                {
                    #region
                    if (string.IsNullOrEmpty(redis.HashGet("pageinfo", url)))
                    {
                        var htmlSourceCode = GetHtmlByUrlAsync(url);
                        var document = parser.ParseDocument(htmlSourceCode.Result);
                        var pageInfo = document.QuerySelector(".co_content8 > .x");
                        string pattern_pagetotal = "(?<=页次：\\d\\/)\\d+";
                        string pattern_pageevery = "(?<=每页)\\d+";
                        string pattern_total = "(?<=总数)\\d+";
                        int page_total = Convert.ToInt32(Regex.Match(pageInfo.OuterHtml, pattern_pagetotal).Value);
                        int page_every = Convert.ToInt32(Regex.Match(pageInfo.OuterHtml, pattern_pageevery).Value);
                        int total = Convert.ToInt32(Regex.Match(pageInfo.OuterHtml, pattern_total).Value);
                        redis.HashSet("pageinfo", url, page_total + "|" + page_every + "|" + total);
                    }
                    #endregion
                }
                Task.Run(() => GetPageList(parser));
            });
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="parser"></param>
        static async Task GetPageList(HtmlParser parser)
        {
            var keys = redis.HashKeys("pageinfo");
            try
            {
                foreach (var key in keys)
                {
                    string url = key.ToString();
                    if (url.EndsWith("index.html"))
                        url = url.Replace("index.html", "");//去掉index.html
                    string pageinfo = redis.HashGet("pageinfo", key);
                    string[] parts = pageinfo.Split('|');
                    int pagetotal = Convert.ToInt32(parts[0]);
                    NLogHelper.Info("当前抓取地址:" + url + ",总页数:" + pagetotal + ",总条数" + Convert.ToInt32(parts[2]));
                    await DoTask(parser, 0, Convert.ToInt32(parts[1]), pagetotal, url, Convert.ToInt32(parts[2]));
                    #region
                    //int average_total = pagetotal / maxTask;
                    //int remainder = pagetotal % maxTask;
                    ////此处可以进行多线程分段下载                
                    //var scheduler = new LimitedConcurrencyLevelTaskScheduler(maxTask);
                    //for(var i = 0; i < maxTask; i++)
                    //{
                    //    int pageIndex = 0 + (i * average_total);
                    //    int pageSize = average_total;
                    //    if (i + 1 == maxTask)
                    //        pageSize = average_total + remainder;
                    //    //Task.Run(() => SplitTask(parser, pageIndex, pageSize, pagetotal, url, Convert.ToInt32(parts[2]), scheduler));
                    //    SplitTask(parser, pageIndex, pageSize, pagetotal, url, Convert.ToInt32(parts[2]), scheduler);
                    //}
                    #endregion
                }
            }
            catch (Exception ex)
            {
                NLogHelper.Error("链接出错，error" + ex.Message);
            }
        }

        static async Task DoTask(HtmlParser parser, int pageindex, int pageSize, int pagetotal, string url, int totalNum)
        {
            try
            {
                Console.WriteLine("Task id {0} Time{1}", Task.CurrentId, DateTime.Now);
                for (int i = pageindex; i < pagetotal; i++)
                {
                    NLogHelper.Info($"当前抓取地址:{url},总页数:{i}/{pagetotal}");
                    if (i > 0)
                    {
                        string pattern = "index_\\d{1,}";
                        Match match = Regex.Match(url, pattern);
                        if (match.Success)
                            url = url.Replace(match.Value, $"index_{i + 1}");
                        else
                            url += $"index_{i + 1}.html";
                    }
                    var html = await GetHtmlByUrlAsync(url);
                    if (string.IsNullOrEmpty(html))
                    {
                        NLogHelper.Info($"{url}解析失败");
                        var error = new Models.ErrorLog()
                        {
                            url = url
                        };
                        using (var db = new Models.MovieContext())
                        {
                            db.ErrorLogs.Add(error);
                            db.SaveChanges();
                        }
                        continue;
                    }
                    var document = parser.ParseDocument(html);
                    var list = document.QuerySelectorAll("td > b");
                    int a_cnt = 1;//列表中的第几条
                    foreach (var item in list)
                    {
                        var a = item.QuerySelector("a");
                        string link = baseurl + a.Attributes["href"]?.Value;
                        StackExchange.Redis.RedisValue value = link;
                        if (!redis.ListRange("downloadlist_ok").Contains(value))
                        {
                            var param = new DownloadParam()
                            {
                                url = link,
                                total = totalNum,
                                page_total = pagetotal,
                                page_size = pageSize,
                                current_item = a_cnt,
                                page_index = i + 1,
                            };
                            await GetPageInfo(param);
                            Thread.Sleep(rd.Next(0, 2000));
                            NLogHelper.Info("download...ok,wait a moment");
                        }
                        else
                            NLogHelper.Warn($"{url}...exist");
                        a_cnt++;
                    }
                    //NLogHelper.Info("翻页，暂停2~5秒..");
                    Thread.Sleep(rd.Next(2000, 5000));
                }
            }
            catch (Exception ex)
            {
                NLogHelper.Error("下载出错:" + ex.Message);
            }
        }

        static async Task GetPageInfo(DownloadParam param)
        {
            ISyncPolicy policy1 = Policy.Handle<Exception>(ex => ex.Message == "解析错误")
                .Fallback(() =>
                {
                    NLogHelper.Debug($"url:{param.url},pagetotal:{param.page_index}/{param.page_total},current_item:{param.current_item}/{param.page_size}...解析失败");
                    var error = new Models.ErrorLog()
                    {
                        url = param.url
                    };
                    using (var db = new Models.MovieContext())
                    {
                        db.ErrorLogs.Add(error);
                        db.SaveChanges();
                    }
                });
            NLogHelper.Debug($"正在下载,url:{param.url},pagetotal:{param.page_index}/{param.page_total},current_item:{param.current_item}/{param.page_size}");

            var html = await GetHtmlByUrlAsync(param.url);
            //var document = parser.ParseDocument(html);
            #region 看瞎眼的正则匹配
            string pattern_score = "(?<=<strong class=\"rank\">)\\d.*(?=</strong>)";//匹配总评分
            string pattern_key = "(?<=<h1>).{1,}(?=</h1>)";//匹配标题（关键字）
            string[] pattern_type = { "(?<=<p>◎类.*别).{1,}(?=</p>)", "(?<=◎类.*别).{1,}(?=<br />◎语.*言)", "(?<=◎语.*言).{1,}(?=<div />)" };//匹配类型            
            string[] pattern_title = { "(?<=<p>◎片.*名).{1,}(?=</p>)", "(?<=◎片.*名).{1,}(?=<br />◎年.*代)", "(?<=◎片.*名).{1,}(?=<div />)" };//匹配片名
            string[] pattern_director = { "(?<=<p>◎导.*演).{1,}(?=</p>)", "(?<=◎导.*演).{1,}(?=<br />◎主.*演)", "(?<=◎导.*演).{1,}(?=<div />)" };//匹配导演
            string[] pattern_relase = { "(?<=<p>◎上映日期).{1,}(?=</p>)", "(?<=◎上映日期).{1,}(?=<br />◎豆瓣评分)", "(?<=◎上映日期).{1,}(?=<div />)" };//匹配上映日期
            string[] pattern_year = { "(?<=<p>◎年.*代).+\\d{4,}(?=</p>)", "(?<=◎年.*代).{1,}(?=<br />◎产.*地)", "(?<=◎年.*代).{1,}(?=<div />)" };//匹配上映日期
            string[] pattern_country = { "(?<=<p>◎产.*地).{1,}(?=</p>)", "(?<=◎产.*地).{1,}(?=<br />◎类.*别)", "(?<=◎主.*演).{1,}(?=<div />)" };//匹配国家
            string[] pattern_douban = { "(?<=<p>◎豆瓣评分).+\\d.+(?=/.+</p>)", "(?<=◎豆瓣评分.)\\d.\\d(?=/)", "(?<=◎豆瓣评分).{1,}(?=<div />)" };//匹配豆瓣评分
            string[] pattern_imdb = { "(?<=<p>◎IMDb评分).+\\d.+(?=/.+</p>)", "(?<=◎IMDb评分.)\\d.\\d(?=/)", "(?<=◎IMDb评分).{1,}(?=<div />)" };//imdb评分
            string[] pattern_mainactor = { "(?<=<p>◎主　　演).{1,}(?=</p>)", "(?<=◎主.*演).*(?=◎简.*介)", "(?<=◎主.*演).{1,}(?=<div />)" };//主要演员
            string magent_link = "(?<=>)magnet.+(?=</a>)";

            var movie = new Models.Movie();
            movie.page_url = param.url;
            movie.score = Regex.Match(html, pattern_score).Success ? Convert.ToDouble(Regex.Match(html, pattern_score).Value) : 0;
            movie.keywords = Regex.Match(html, pattern_key).Success ? NoHTML(Convert.ToString(Regex.Match(html, pattern_key).Value)) : "";
            foreach (string pattern in pattern_type)
            {
                if (Regex.Match(html, pattern).Success)
                {
                    movie.type = NoHTML(Convert.ToString(Regex.Match(html, pattern).Value));
                    break;
                }
            }
            foreach (string pattern in pattern_title)
            {
                if (Regex.Match(html, pattern).Success)
                {
                    movie.title = NoHTML(Convert.ToString(Regex.Match(html, pattern).Value));
                    break;
                }
            }
            foreach (string pattern in pattern_director)
            {
                if (Regex.Match(html, pattern).Success)
                {
                    movie.director = NoHTML(Convert.ToString(RemoveOverNull(Regex.Match(html, pattern).Value.Replace("<br />", ""))));
                    break;
                }
            }
            foreach (string pattern in pattern_relase)
            {
                if (Regex.Match(html, pattern).Success)
                {
                    movie.relase_time = NoHTML(Convert.ToString(Regex.Match(html, pattern).Value));
                    break;
                }
            }
            foreach (string pattern in pattern_year)
            {
                if (Regex.Match(html, pattern).Success)
                {
                    movie.year = Convert.ToString(Regex.Match(html, pattern).Value);
                    break;
                }
            }
            foreach (string pattern in pattern_country)
            {
                if (Regex.Match(html, pattern).Success)
                {
                    movie.country = NoHTML(Convert.ToString(Regex.Match(html, pattern).Value));
                    break;
                }
            }
            foreach (string pattern in pattern_douban)
            {
                if (Regex.Match(html, pattern).Success)
                {
                    movie.douban = Convert.ToDouble(Regex.Match(html, pattern).Value.Trim());
                    break;
                }
            }
            foreach (string pattern in pattern_imdb)
            {
                if (Regex.Match(html, pattern).Success)
                {
                    movie.imdb = Convert.ToDouble(Regex.Match(html, pattern).Value.Trim());
                    break;
                }
            }
            foreach (string pattern in pattern_mainactor)
            {
                if (Regex.Match(html, pattern).Success)
                {
                    movie.main_actor = NoHTML(Convert.ToString(Regex.Match(html, pattern).Value.Replace("<br />", "")));
                    break;
                }
            }
            List<Models.Resource> list = new List<Models.Resource>();
            foreach (Match match in Regex.Matches(html, magent_link))
            {
                var resource = new Models.Resource();
                resource.url = match.Value;
                resource.movie = movie;
                list.Add(resource);
            }
            #endregion
            using (var db = new Models.MovieContext())
            {
                if (db.Movies.Count(u => u.keywords == movie.keywords) == 0)
                {
                    await db.Movies.AddAsync(movie);
                    await db.Resources.AddRangeAsync(list);
                    await db.SaveChangesAsync();
                    NLogHelper.Debug($"{param.url}...download ok");
                    await redis.ListLeftPushAsync("downloadlist_ok", param.url);
                }
                else
                {

                    NLogHelper.Warn($"{param.url}...已存在");
                }
            }
        }

        public static string RemoveOverNull(string str)
        {
            string[] arr = str.Trim().Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
            string result = string.Join(" ", arr);
            return result;
        }

        public static string NoHTML(string Htmlstring)
        {
            //删除脚本
            Htmlstring = Regex.Replace(Htmlstring, @"<script[^>]*?>.*?</script>", "",
               RegexOptions.IgnoreCase);
            //删除HTML
            Htmlstring = Regex.Replace(Htmlstring, @"<(.[^>]*)>", "",
             RegexOptions.IgnoreCase);
            Htmlstring = Regex.Replace(Htmlstring, @"([\r\n])[\s]+", "",
             RegexOptions.IgnoreCase);
            Htmlstring = Regex.Replace(Htmlstring, @"-->", "", RegexOptions.IgnoreCase);
            Htmlstring = Regex.Replace(Htmlstring, @"<!--.*", "", RegexOptions.IgnoreCase);
            Htmlstring = Regex.Replace(Htmlstring, @"&(quot|#34);", "\"",
             RegexOptions.IgnoreCase);
            Htmlstring = Regex.Replace(Htmlstring, @"&(amp|#38);", "&",
             RegexOptions.IgnoreCase);
            Htmlstring = Regex.Replace(Htmlstring, @"&(lt|#60);", "<",
             RegexOptions.IgnoreCase);
            Htmlstring = Regex.Replace(Htmlstring, @"&(gt|#62);", ">",
               RegexOptions.IgnoreCase);
            Htmlstring = Regex.Replace(Htmlstring, @"&(nbsp|#160);", "   ",
               RegexOptions.IgnoreCase);
            Htmlstring = Regex.Replace(Htmlstring, @"&middot;", "",
              RegexOptions.IgnoreCase);
            Htmlstring = Regex.Replace(Htmlstring, @"&(iexcl|#161);", "\xa1",
              RegexOptions.IgnoreCase);
            Htmlstring = Regex.Replace(Htmlstring, @"&(cent|#162);", "\xa2",
              RegexOptions.IgnoreCase);
            Htmlstring = Regex.Replace(Htmlstring, @"&(pound|#163);", "\xa3",
              RegexOptions.IgnoreCase);
            Htmlstring = Regex.Replace(Htmlstring, @"&(copy|#169);", "\xa9",
             RegexOptions.IgnoreCase);
            Htmlstring = Regex.Replace(Htmlstring, @"&#(\d+);", "",
             RegexOptions.IgnoreCase);

            Htmlstring.Replace("<", "");
            Htmlstring.Replace(">", "");
            Htmlstring.Replace("\r\n", "");
            return Htmlstring;
        }
        #endregion
    }
}
